import scrapy

many = 0


class PengpaiSpider(scrapy.Spider):
    name = 'pengpai'
    allowed_domains = ['thepaper.cn']
    start_urls = [
        'https://m.thepaper.cn/list_page.jsp?nodeid=25600&isList=1&pageidx=2&lastTime=1604981182532&mrd=0.6879094976816258&filterIds=']
    page_num = 2

    def parse(self, response):
        # print('==========>', response)
        div_list = response.xpath('/html/body/div[@class="list_item"]')
        print(len(div_list))
        for div_temp in div_list:
            global many

            if many <= 100:
                print(many)
                many += 1
                new_title = div_temp.xpath('./div/div[1]/span/a/text()').extract_first()
                new_img = div_temp.xpath('./a/img/@src').extract_first()
                new_url = div_temp.xpath('./a/@href').extract_first()
                new_time = div_temp.xpath('./span[2]/text()').extract_first()
                # new_text = new_url.xpath('//*[@id="current_desc"]/text()').extract_first()
                print(new_title, new_img, new_url, new_time)

                yield {
                    'type': 'info',
                    'new_title': new_title,
                    'new_img': new_img,
                    'new_url': new_url,
                    'new_time': new_time
                }
                # 生成Request，爬取图片
                yield scrapy.Request(url=new_img, callback=self.parse_img, cb_kwargs={"img_name": new_title})
            self.page_num += 1
            if self.page_num <= 5:
                # print(self.page_num)
                next_page_url = 'https://m.thepaper.cn/list_page.jsp?nodeid=25600&isList=1&pageidx=%s&lastTime=1604981182532&mrd=0.6879094976816258&filterIds=' % self.page_num
                # 生成Request，爬取下一页的数据
                yield scrapy.Request(url=next_page_url, callback=self.parse)

    def parse_img(self, response, img_name):
        print("----图片url----", response.url)
        yield {
            'type': "img",
            'img_name': img_name + ".png",
            'img_bytes': response.body
        }
